import os
import re
import time
import urllib.parse as parse

import facade
import requests
from bs4 import BeautifulSoup
from xjlibrary.mrequest import baserequest
from xjlibrary.our_file_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -3)
sPath = BaseDir.get_new_path(TopPath, "download", "cnki_cg", "download")
HtmlYearRoot = os.path.join(sPath, 'year_html')
HtmlSubClass = os.path.join(sPath, 'subclass_html')

class_dict = {'A': '基础科学', 'B': '工程科技Ⅰ辑', 'C': '工程科技Ⅱ辑', 'D': '农业科技', 'E': '医药卫生科技', 'F': '哲学与人文科学', 'G': '社会科学Ⅰ辑',
              'H': '社会科学Ⅱ辑', 'I': '信息科技', 'J': '经济与管理科学'}

ListSqls = list()
nCount = 0
mysqlutils = facade.MysqlUtiles(BaseDir.get_new_path(curPath, "db.ini"), "db", facade.get_streamlogger())

# Proxiesss = {
#     'http': '127.0.0.1:8087',
#     # 'http':'162.105.138.192:8092',
#     'https': '127.0.0.1:8087'  # key是指目标网站的协议
# }

UserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'


# 根据分类代号下载年份目录 输入参数: NaviCode 成果子分类代号
def DownYear(NaviCode):
    if not os.path.exists(HtmlYearRoot):
        os.makedirs(HtmlYearRoot)
    outPathFile = os.path.join(HtmlYearRoot, NaviCode + '.htm')
    if os.path.exists(outPathFile):
        print("文件存在"+outPathFile)
        return True
    r = None
    sMsg = None
    sn = requests.Session()
    sn.headers['User-Agent'] = UserAgent
    url = 'http://kns.cnki.net/KNS/brief/result.aspx?dbprefix=SNAD'
    print('input url:%s' % url)
    try:
        r = sn.get(url,
                   # proxies=Proxies,
                   timeout=30)
    except:
        print('net error')
        sMsg = '1'

    if sMsg:
        return False

    sMsg = None
    url = 'http://kns.cnki.net/kns/request/SearchHandler.ashx?action='
    url += '&NaviCode=' + NaviCode
    url += '&catalogName=ZJCLS&ua=1.25&PageName=ASP.brief_result_aspx&DbPrefix=SNAD&DbCatalog=国家科技成果数据库&ConfigFile=SNAD.xml&db_opt=SNAD&db_value=国家科技成果数据库&his=0'
    print('input url:%s' % url)
    try:
        r = sn.get(url,
                   #proxies=Proxies,
                    timeout=30)
    except:
        print('net error')
        sMsg = '1'

    if sMsg:
        return False
    # print(r)
    # url =
    # '''http://kns.cnki.net/kns/group/doGroupLeft.aspx?action=1&Param=ASP.brief_result_aspx#CISD/年/年,count(*)/年/(年,'date')#年$desc/1000000$/-/40/40000/ButtonView&cid=0&clayer=0'''
    sMsg = None
    url = 'http://kns.cnki.net/KNS/group/doGroupLeft.aspx?action=1&Param=ASP.brief_result_aspx%23SNAD/%u5E74/%u5E74%2Ccount%28*%29/%u5E74/%28%u5E74%2C%27date%27%29%23%u5E74%24desc/1000000%24/-/40/40000/ButtonView'

    # req = requests.Request('GET', url)
    # prepped = sn.prepare_request(req)
    # prepped.url = prepped.url.replace('%25', '%')
    # r = sn.send(prepped)

    url = 'http://kns.cnki.net/kns/group/doGroupLeft.aspx?' + \
          parse.urlencode(dict(parse.parse_qsl(parse.urlparse(url).query)))
    print('input url:%s' % url)
    try:
        r = sn.get(url,
                   # proxies=Proxies,
                   timeout=20)
    except:
        print('net error')
        sMsg = '1'

    if sMsg:
        return False
    # print(r)
    # print(r.url)

    print('outPathFile:%s' % outPathFile)
    with open(outPathFile, mode='wb') as f:
        f.write(r.content)

    return True


# 根据成果根分类下载子分类页面 输入参数: navicode 成果类型代号


def GetSubClassId(NaviCode):
    outPathFile = os.path.join(HtmlSubClass, NaviCode + '.htm')
    if BaseDir.is_file_exists(outPathFile):
        print("文件存在"+outPathFile)
        return True
    sn = requests.Session()
    sn.headers['User-Agent'] = UserAgent
    url = 'http://kns.cnki.net/KNS/brief/result.aspx?dbprefix=SNAD'
    BoolResult, errString, r = baserequest.BaseRequest(url,
                                                       sn,
                                                       # proxies=Proxies,
                                                       endstring="",
                                                       timeout=30
                                                       )
    if not BoolResult:
        return False

    timestring = time.strftime("%a %b %d %Y %H:%M:%S ")
    timestring = "{}{}({})".format(parse.quote(timestring), parse.quote("GMT+0800 "), parse.quote("中国标准时间"))
    print(timestring)

    sMsg = None
    # http://kns.cnki.net/KNS/request/NaviGroup.aspx?code=A&tpinavigroup=SNADtpiresult&catalogName=ZJCLS
    url = 'http://kns.cnki.net/KNS/request/NaviGroup.aspx'
    url += '?code=' + NaviCode
    url += '&tpinavigroup=SNADtpiresult&catalogName=ZJCLS'
    print('input url:%s' % url)
    BoolResult, errString, r = baserequest.BaseRequest(url,
                                                       sn,
                                                       #proxies=Proxies,
                                                       endstring="",
                                                       timeout=30,
                                                       )
    if not BoolResult:
        return False

    if not os.path.exists(HtmlSubClass):
        os.makedirs(HtmlSubClass)

    with open(outPathFile, mode='wb') as f:
        f.write(r.content)
    return True


def SaveSubClass(sPath):
    global ListSqls
    print('Start!!!')
    soup = BeautifulSoup(open(sPath, 'r', encoding='utf-8'), "lxml")
    aLinks = soup.find_all('a')
    for a in aLinks:
        subid = ''
        subname = ''
        node = a.get('onclick')
        m = re.match(r"ClickNode\('(\S+)','(\S+)'\);", node)
        if m:
            subid = m.groups()[0]
            subname = "".join(a.stripped_strings)
            sql = "replace into subclass(subid, subname) VALUES('{subid}', '{subname}')".format(
                subid=subid, subname=subname)
            ListSqls.append(sql)

    InsertIntoDbFromList()


# 插入数据库


def InsertIntoDbFromList():
    global nCount, ListSqls
    mysqlutils.ExeSqlListToDB(ListSqls)
    ListSqls = list()


# 读取二级目录id
def ReadNaviCodeFromDb():
    sql = 'SELECT subid FROM subclass'
    rows = mysqlutils.SelectFromDB(sql)
    return rows


# 下载二级目录
def DownLoadSubCode():
    # 按分类下载后保存到 subclass_html 目录
    for keys in sorted(class_dict.keys()):
        while 1:
            print('Parent:' + keys)
            nResult = GetSubClassId(keys)
            if nResult:
                break
    # 解析目录下的数据
    subPath = os.path.join(sPath, 'subclass_html')
    print('path:' + subPath)
    # 解析出分类名和编号
    for path, dirNames, fileNames in os.walk(subPath):
        for fileName in fileNames:
            filePath = os.path.join(path, fileName)
            print(filePath)
            # AnalysisFile(filePath,fileName)
            SaveSubClass(filePath)


def DoWork():
    StartTime = time.time()
    DownLoadSubCode()
    taskList = ReadNaviCodeFromDb()
    if taskList:
        for task in taskList:
            while 1:
                print('NaviCode:' + task[0])
                nResult = DownYear(task[0])
                if nResult:
                    break

    print('Time total:' + repr(time.time() - StartTime) + '\n')


def main(logger=None):
    StartTime = time.time()
    DownLoadSubCode()
    taskList = ReadNaviCodeFromDb()
    if taskList:
        for task in taskList:
            while 1:
                print('NaviCode:' + task[0])
                nResult = DownYear(task[0])
                if nResult:
                    break

    print('Time total:' + repr(time.time() - StartTime) + '\n')


if __name__ == '__main__':
    main()
